/*
 * License Agreement for OpenSearchServer
 *
 * Copyright (C) 2010-2017 Emmanuel Keller / Jaeksoft
 * 
 * http://www.open-search-server.com
 * 
 * This file is part of OpenSearchServer.
 *
 * OpenSearchServer is free software: you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 * OpenSearchServer is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with OpenSearchServer. 
 *  If not, see <http://www.gnu.org/licenses/>.
 */

package com.jaeksoft.searchlib.parser;

import com.jaeksoft.searchlib.ClientCatalog;
import com.jaeksoft.searchlib.Logging;
import com.jaeksoft.searchlib.SearchLibException;
import com.jaeksoft.searchlib.analysis.ClassPropertyEnum;
import com.jaeksoft.searchlib.analysis.LanguageEnum;
import com.jaeksoft.searchlib.ocr.HocrDocument;
import com.jaeksoft.searchlib.ocr.HocrPdf;
import com.jaeksoft.searchlib.ocr.HocrPdf.HocrPage;
import com.jaeksoft.searchlib.ocr.OcrManager;
import com.jaeksoft.searchlib.streamlimiter.StreamLimiter;
import com.jaeksoft.searchlib.util.ExecuteUtils.ExecutionException;
import com.jaeksoft.searchlib.util.GhostScript;
import com.jaeksoft.searchlib.util.IOUtils;
import com.jaeksoft.searchlib.util.ImageUtils;
import com.jaeksoft.searchlib.util.PdfCrack;
import com.jaeksoft.searchlib.util.StringUtils;
import com.jaeksoft.searchlib.util.ThreadUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.comparator.LastModifiedFileComparator;
import org.apache.pdfbox.io.MemoryUsageSetting;
import org.apache.pdfbox.multipdf.PDFMergerUtility;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageTree;
import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
import org.apache.pdfbox.text.PDFTextStripper;

import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;

public class PdfParser extends Parser {

	public static final String[] DEFAULT_MIMETYPES = { "application/pdf" };

	public static final String[] DEFAULT_EXTENSIONS = { "pdf" };

	public static final Semaphore gsSemaphore = new Semaphore(Runtime.getRuntime().availableProcessors());

	private static ParserFieldEnum[] fl = { ParserFieldEnum.parser_name,
			ParserFieldEnum.title,
			ParserFieldEnum.author,
			ParserFieldEnum.subject,
			ParserFieldEnum.content,
			ParserFieldEnum.producer,
			ParserFieldEnum.keywords,
			ParserFieldEnum.creation_date,
			ParserFieldEnum.modification_date,
			ParserFieldEnum.language,
			ParserFieldEnum.number_of_pages,
			ParserFieldEnum.ocr_content,
			ParserFieldEnum.image_ocr_boxes,
			ParserFieldEnum.pdfcrack_password };

	public PdfParser() {
		super(fl);
	}

	@Override
	public void initProperties() throws SearchLibException {
		super.initProperties();
		addProperty(ClassPropertyEnum.SIZE_LIMIT, "0", null, 20, 1);
		addProperty(ClassPropertyEnum.GHOSTSCRIPT_BINARYPATH, "", null, 50, 1);
		addProperty(ClassPropertyEnum.PDFCRACK_COMMANDLINE, "", null, 50, 1);
	}

	private Calendar getCreationDate(PDDocumentInformation pdfInfo) {
		try {
			return pdfInfo.getCreationDate();
		} catch (Exception e) {
			Logging.warn(e);
			return null;
		}
	}

	private Calendar getModificationDate(PDDocumentInformation pdfInfo) {
		try {
			return pdfInfo.getCreationDate();
		} catch (Exception e) {
			Logging.warn(e);
			return null;
		}
	}

	private String getDate(Calendar cal) {
		return cal == null ? null : cal.getTime().toString();
	}

	private void extractMetaData(ParserResultItem result, PDDocument pdf) throws IOException {
		PDDocumentInformation info = pdf.getDocumentInformation();
		if (info != null) {
			result.addField(ParserFieldEnum.title, info.getTitle());
			result.addField(ParserFieldEnum.subject, info.getSubject());
			result.addField(ParserFieldEnum.author, info.getAuthor());
			result.addField(ParserFieldEnum.producer, info.getProducer());
			result.addField(ParserFieldEnum.keywords, info.getKeywords());
			String d = getDate(getCreationDate(info));
			if (d != null)
				result.addField(ParserFieldEnum.creation_date, d);
			d = getDate(getModificationDate(info));
			if (d != null)
				result.addField(ParserFieldEnum.modification_date, d);
		}
		int pages = pdf.getNumberOfPages();
		result.addField(ParserFieldEnum.number_of_pages, pages);
		PDDocumentCatalog catalog = pdf.getDocumentCatalog();
		if (catalog != null) {
			result.addField(ParserFieldEnum.language, catalog.getLanguage());
		}
	}

	private int addLine(ParserResultItem result, String line) {
		if (line == null)
			return 0;
		line = StringUtils.replaceConsecutiveSpaces(line, " ").trim();
		int l = line.length();
		if (l == 0)
			return 0;
		result.addField(ParserFieldEnum.content, line);
		return line.length();
	}

	/**
	 * Extract text content using PDFBox
	 *
	 * @param result
	 * @param pdf
	 * @throws IOException
	 */
	private int extractTextContent(ParserResultItem result, PDDocument pdf) throws IOException {
		final PDFTextStripper stripper = new PDFTextStripper();
		final String text = stripper.getText(pdf);
		if (StringUtils.isEmpty(text))
			return 0;
		final String[] lines = StringUtils.splitLines(text);
		int characterCount = 0;
		for (String line : lines)
			characterCount += addLine(result, line);
		return characterCount;
	}

	/**
	 * Extract text content using Ghostscript
	 *
	 * @param result
	 * @param context
	 * @throws IOException
	 * @throws InterruptedException
	 */
	private int extractTextContent(ParserResultItem result, PdfOcrContext context)
			throws IOException, InterruptedException {
		File textFile = null;
		BufferedReader bufferedReader = null;
		FileReader fileReader = null;
		try {
			textFile = File.createTempFile("oss_pdfparser", "txt");
			context.ghostScript.extractText(context.pdfPassword, context.pdfFile, textFile);
			fileReader = new FileReader(textFile);
			bufferedReader = new BufferedReader(fileReader);
			int characterCount = 0;
			String line;
			while ((line = bufferedReader.readLine()) != null)
				characterCount += addLine(result, line);
			return characterCount;
		} catch (ExecutionException e) {
			Logging.warn("Ghostscript returned: " + e.getReturnedText());
			throw e;
		} finally {
			IOUtils.close(bufferedReader, fileReader);
			if (textFile != null)
				if (textFile.exists())
					textFile.delete();
		}
	}

	private String pdfCrack(File pdfFile) {
		// New attempt with PDFCrack
		final String pdfCrackCommandLine = getStringProperty(ClassPropertyEnum.PDFCRACK_COMMANDLINE);
		if (StringUtils.isEmpty(pdfCrackCommandLine))
			return null;
		try {
			return PdfCrack.findPassword(pdfCrackCommandLine, pdfFile);
		} catch (IOException e) {
			Logging.warn("PdfCrawl failure", e);
			return null;
		}
	}

	@Override
	protected void parseContent(StreamLimiter streamLimiter, final LanguageEnum lang) throws IOException {
		final PdfOcrContext context = new PdfOcrContext();
		context.lang = lang;
		String fileName = null;
		try {
			final String ghostScriptBinaryPath = getStringProperty(ClassPropertyEnum.GHOSTSCRIPT_BINARYPATH);
			context.ghostScript = StringUtils.isEmpty(ghostScriptBinaryPath) ? null : new GhostScript(
					ghostScriptBinaryPath);
			fileName = streamLimiter.getFile().getName();
			context.pdfFile = streamLimiter.getFile();
			try {
				context.pdf = PDDocument.load(context.pdfFile);
			} catch (InvalidPasswordException e) {
				context.pdfPassword = pdfCrack(context.pdfFile);
				if (context.pdfPassword == null) {
					Logging.warn("PDFBox decryption failed " + fileName);
					IOUtils.closeQuietly(context.pdf);
					context.pdf = null;
				}
			}

			final ParserResultItem result = getNewParserResultItem();
			result.addField(ParserFieldEnum.pdfcrack_password, context.pdfPassword);
			if (context.pdf != null)
				extractMetaData(result, context.pdf);
			int charCount = 0;
			if (context.ghostScript == null) {
				if (context.pdf != null)
					charCount = extractTextContent(result, context.pdf);
			} else
				charCount = extractTextContent(result, context);
			if (charCount == 0 && context.pdf != null)
				extractImagesForOCR(result, context);
			result.langDetection(10000, ParserFieldEnum.content);
		} catch (SearchLibException | InterruptedException | java.util.concurrent.ExecutionException e) {
			throw new IOException("Failed on " + fileName, e);
		} finally {
			if (context.pdf != null)
				context.pdf.close();
		}
	}

	private HocrDocument doOcr(OcrManager ocr, LanguageEnum lang, BufferedImage image)
			throws IOException, InterruptedException, SearchLibException {
		File hocrFile = null;
		try {
			hocrFile = File.createTempFile("ossocr", "." + ocr.getHocrFileExtension());
			ocr.ocerizeImage(image, hocrFile, lang, true);
			if (hocrFile.length() == 0)
				return null;
			return new HocrDocument(hocrFile);
		} finally {
			if (hocrFile != null)
				FileUtils.deleteQuietly(hocrFile);
		}
	}

	private HocrDocument doOcr(OcrManager ocr, LanguageEnum lang, File imageFile)
			throws IOException, InterruptedException, SearchLibException {
		File hocrFile = null;
		try {
			hocrFile = File.createTempFile("ossocr", "." + ocr.getHocrFileExtension());
			ocr.ocerize(imageFile, hocrFile, lang, true);
			if (hocrFile.length() == 0)
				return null;
			return new HocrDocument(hocrFile);
		} finally {
			if (hocrFile != null)
				FileUtils.deleteQuietly(hocrFile);
		}
	}

	private void ocrImageGhostcript(PdfOcrContext context, int page)
			throws IOException, InterruptedException, SearchLibException {
		File imageFile = null;
		try {
			imageFile = File.createTempFile("oss_pdfparser", ".png");
			gsSemaphore.acquire();
			try {
				context.ghostScript.generateImage(context.pdfPassword, page, context.pdfFile, 300, imageFile);
			} finally {
				gsSemaphore.release();
			}
			Dimension dimension = ImageUtils.getDimensions(imageFile);
			HocrPage hocrPage = context.hocrPdf.createPage(page - 1, dimension.width, dimension.height);
			hocrPage.addImage(doOcr(context.ocr, context.lang, imageFile));
		} finally {
			if (imageFile != null)
				if (imageFile.exists())
					imageFile.delete();
		}
	}

	public class PdfOcrContext {

		private PDDocument pdf = null;
		private OcrManager ocr = null;
		private LanguageEnum lang = null;
		private PDFRenderer pdfRenderer = null;
		private GhostScript ghostScript = null;
		private File pdfFile = null;
		private String pdfPassword = null;
		private HocrPdf hocrPdf = null;
	}

	public class ImageOcrCallable implements Callable<Boolean> {

		private final PdfOcrContext context;
		private final int currentPage;
		private final AtomicInteger emptyPageImages;

		public ImageOcrCallable(PdfOcrContext context, int currentPage, AtomicInteger emptyPageImages) {
			this.context = context;
			this.currentPage = currentPage;
			this.emptyPageImages = emptyPageImages;
		}

		@Override
		public Boolean call() throws IOException, InterruptedException, SearchLibException {
			if (context.ghostScript == null) {
				final BufferedImage image = context.pdfRenderer.renderImageWithDPI(currentPage, 300, ImageType.RGB);
				if (ImageUtils.checkIfManyColors(image)) {
					final HocrPage hocrPage = context.hocrPdf.createPage(currentPage, image.getWidth(),
							image.getHeight());
					hocrPage.addImage(doOcr(context.ocr, context.lang, image));
				} else
					emptyPageImages.incrementAndGet();
			} else {
				ocrImageGhostcript(context, currentPage + 1);
			}
			return true;
		}
	}

	private void extractImagesForOCR(ParserResultItem result, PdfOcrContext context)
			throws SearchLibException, IOException, InterruptedException, java.util.concurrent.ExecutionException {

		context.ocr = ClientCatalog.getOcrManager();
		if (context.ocr == null || context.ocr.isDisabled())
			return;
		if (!getFieldMap().isMapped(ParserFieldEnum.ocr_content) && !getFieldMap().isMapped(
				ParserFieldEnum.image_ocr_boxes))
			return;
		if (context.ghostScript == null)
			context.pdfRenderer = new PDFRenderer(context.pdf);

		context.hocrPdf = new HocrPdf();
		final PDPageTree pageTree = context.pdf.getDocumentCatalog().getPages();
		int currentPage = 0;
		AtomicInteger emptyPageImages = new AtomicInteger(0);

		final ExecutorService executorService = config.getThreadPool();
		final List<Future<Boolean>> futures = new ArrayList<>();
		for (PDPage page : pageTree) {
			final ImageOcrCallable callable = new ImageOcrCallable(context, currentPage++, emptyPageImages);
			futures.add(executorService.submit(callable));
		}
		ThreadUtils.<Boolean>done(futures);

		if (currentPage > 0 && emptyPageImages.get() == currentPage)
			throw new SearchLibException("All pages are blank " + currentPage);

		if (getFieldMap().isMapped(ParserFieldEnum.image_ocr_boxes))
			context.hocrPdf.putHocrToParserField(result, ParserFieldEnum.image_ocr_boxes);
		if (getFieldMap().isMapped(ParserFieldEnum.ocr_content))
			context.hocrPdf.putTextToParserField(result, ParserFieldEnum.ocr_content);

	}

	@Override
	public void mergeFiles(File fileDir, File destFile) throws SearchLibException {
		PDFMergerUtility pdfMerger = new PDFMergerUtility();
		final File[] files = new LastModifiedFileComparator().sort(fileDir.listFiles());
		for (File file : files) {
			String ext = FilenameUtils.getExtension(file.getName());
			if (!"pdf".equalsIgnoreCase(ext))
				continue;
			try {
				pdfMerger.addSource(file);
			} catch (FileNotFoundException e) {
				throw new SearchLibException(e);
			}
		}
		if (destFile.exists())
			destFile.delete();
		pdfMerger.setDestinationFileName(destFile.getAbsolutePath());
		try {
			pdfMerger.mergeDocuments(MemoryUsageSetting.setupMixed(100 * 1024 * 1024));
		} catch (IOException e) {
			throw new SearchLibException(e);
		}
	}
}
